package parsers;
/*
 * Maybe it needs to send the cookie back to the server so that it knows?
 */
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.URL;
import java.sql.Date;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.Tag;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.NotFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.nodes.TagNode;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.ScriptTag;
import org.htmlparser.util.NodeList;
import org.jdom.Attribute;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;
import org.jdom.input.SAXBuilder;
import org.jdom.output.Format;
import org.jdom.output.XMLOutputter;

import com.sun.xml.internal.ws.transport.http.client.HttpCookie;

import constants.VeritasHtmlConstants;
import constants.XML_Namespaces;

public class VertisParse extends defaultParse{

	
	public static final char sep = File.separatorChar;

	public VertisParse(String strName, int strID, int adID)
	{
		storeName = strName;
		storeId= strID;
		adId=adID;
	}
	
	@Override
	public String nextURL() {
		// TODO Auto-generated method stub
		return null;
	}

	@Override
	public boolean outputXML() {
		// TODO Auto-generated method stub
		return false;
	}

	@Override
	/*
	 * Give a file name that contains the product listing from the ad page.
	 * This is HTML between the comments <!-- Items: Start --> and <!-- Items: End -->
	 * Returns a file name containing the categories and URL information in XML
	 * following the schema specified in Detail_CategoryOutputFormat.xsd.
	 */
	public String getDetailURLsAndCategories(String filename, String baseURL, String pageNum) {
		//File content = new File(filename);
				
		
		AndFilter anchor = new AndFilter(new TagNameFilter("a"), new NotFilter(new HasAttributeFilter("id", null)));
		//attempted to find a better filter so that only scripts containing categories were extracted.
		//AndFilter script = new AndFilter(new TagNameFilter("script"),new StringFilter("cat["));
		OrFilter filter = new OrFilter(new TagNameFilter("script"),anchor);//new TagNameFilter("script")script
		String output="." + sep + "Test_Output" + sep + "TestOutput.xml";
		try
		{
			Parser parse = new Parser(filename);
			
			NodeList list = parse.extractAllNodesThatMatch(filter);
			
			FileOutputStream fos = new FileOutputStream(output);
			
			
			
			XMLOutputter xmlOut = new XMLOutputter();
			Document outDoc = new Document();
			
			Element root = new Element("Items");
			
			Namespace outputNS= Namespace.getNamespace("tns", constants.XML_Namespaces.Detail_CategoryOutputFormat);
			root.setNamespace(outputNS);
			Namespace xsiNS = Namespace.getNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
			root.setAttribute(new Attribute("schemaLocation", "Detail_CategoryOutputFormat ../XML_Formats/Detail_CategoryOutputFormat.xsd",xsiNS));
			root.setAttribute(new Attribute("Date",""));
			root.setAttribute(new Attribute("pageNum",pageNum));
			root.setAttribute(new Attribute("StoreName",storeName));
			
			outDoc.setRootElement(root);
			
			//TODO comment out .setFormat line to remove whitespace
			xmlOut.setFormat(Format.getPrettyFormat());

			
			
			Element detailURL = new Element("detailURL").addContent("value");
			detailURL.setNamespace(outputNS);

			
			NodeList scriptList = list.extractAllNodesThatMatch(new TagNameFilter("script"));
			list.keepAllNodesThatMatch(new TagNameFilter("a"));
			System.out.println(list.size());
			for (int a = 0; a< list.size(); a++)
			{
		//		System.out.println(list.elementAt(a).toHtml());
				Element i = new Element("Item");//.setContent(detailURL);
				i.setNamespace(outputNS);
				root.addContent(i);

				
				LinkTag link = (LinkTag) list.elementAt(a);
				i.addContent(parseLinkTag(link,baseURL));
				
				String scriptCode =null;
				ScriptTag scriptTag = null;
				do
				{
					scriptTag = (ScriptTag) scriptList.remove(0);
					scriptCode= scriptTag.getScriptCode();
				}while(!scriptCode.startsWith("cat["));
				
				i.addContent(parseScriptTag(scriptTag));
				
			
					
				
			}
			xmlOut.output(outDoc, fos);
			
			/*if(list.size()>0)
				System.out.println(list.elementAt(0).toString());*/
			fos.close();
			

			
			
		}
		catch (Exception e){System.out.println("Parse exception: "+e.toString());}
		finally
		{
			
		}
		return output;
	}

	// returns a single tns:detailURL element to add to an item Element
	private Element parseLinkTag(LinkTag link, String baseURL) {
		// TODO Auto-generated method stub
		
		Namespace outputNS= Namespace.getNamespace("tns", constants.XML_Namespaces.Detail_CategoryOutputFormat);
		
		String onClickText = link.getAttribute("onclick");
		String onClickParameters = link.getAttribute("onclick").substring(onClickText.indexOf('(')+1, onClickText.lastIndexOf(')'));
		System.out.println(onClickParameters);
		String parameterList[] = onClickParameters.trim().replaceAll("\"","").split(",");
		
		//assumes last URL_ARGUEMENT value is for page #1
		String  Url_Arguements[]  =  constants.URL_Variables.URL_ARGUEMENTS;
		for(int a = 0; a <Url_Arguements.length-1; a++)
		{
			String urlArg =Url_Arguements[a];
			
			///how to find out which element of paramterList to use
			int urlParameterIndex = new Integer(urlArg.substring(urlArg.indexOf("ARG")+3,urlArg.indexOf("\\}"))).intValue();
			baseURL = baseURL.replaceAll(urlArg, parameterList[urlParameterIndex].trim());
		}
		String urlArg =Url_Arguements[Url_Arguements.length-1];
		if(parameterList.length != Url_Arguements.length)
			baseURL = baseURL.replaceAll(urlArg, "1");
		else
		{
			int urlParameterIndex = new Integer(urlArg.substring(urlArg.indexOf("ARG"),urlArg.indexOf('}'))).intValue();
			
			baseURL = baseURL.replaceAll(urlArg, parameterList[urlParameterIndex]);
		}
		
		
		return new Element("detailURL",outputNS).addContent(baseURL);
	}

	//returns a collection of Category XML Elements to add to an item Element
	private Collection <Element> parseScriptTag(ScriptTag script) {
		// TODO Auto-generated method stub
		Namespace outputNS= Namespace.getNamespace("tns", constants.XML_Namespaces.Detail_CategoryOutputFormat);
		String scriptCode = script.getScriptCode().replaceAll("\"","");
		String htmlCategories =scriptCode.substring(
				scriptCode.indexOf(',')+1, scriptCode.lastIndexOf(')'));

		String [] categoryList = htmlCategories.split(",");
		Collection <Element> categories =new ArrayList<Element>();
		

		for(int b = 0; b<categoryList.length; b++)
		{
			categories.add(new Element("category",outputNS).addContent(categoryList[b]));
			System.out.println(categoryList[b]);
		}
		return categories;
	}

	@Override
	/*
	 * Given a filename that contains the URLs for detail information of products and
	 * categories following Detail_CategoryOutputFormat.xsd.  This method will open 
	 * the url and parse the details.  Returns a file name of an XML document
	 *  containing the information from the details page of each item in the
	 *  file filename.
	 */
	public String getProductDetails(String filename) {
		Namespace docNamespace= Namespace.getNamespace("tns", constants.XML_Namespaces.Detail_CategoryOutputFormat);
		File file = new File(filename);
		Document doc=null;
		SAXBuilder build = new SAXBuilder();
		XMLOutputter xmlOut = new XMLOutputter();
		Document outDoc = new Document();
		Element root = new Element("ItemDetails");
		Namespace outputNS= Namespace.getNamespace("tns", constants.XML_Namespaces.ItemDetailOutputFormat);
		root.setNamespace(outputNS);
		Namespace xsiNS = Namespace.getNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
		root.setAttribute(new Attribute("schemaLocation", "ItemDetailOutputFormat ../XML_Formats/ItemDetailOutputFormat.xsd",xsiNS));
		root.setAttribute(new Attribute("Date",""));
		root.setAttribute(new Attribute("pageNum",""));
		root.setAttribute(new Attribute("StoreName",storeName));
		
		try {
			doc = build.build(file);
		} catch (JDOMException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}		
	
		List <Element>childrena = doc.getRootElement().getChildren();
		
		//String storeName = doc.getRootElement().getAttribute("StoreName").getValue();
		
		String httpCookie = "GIVE ME COOKIE";
		for(ListIterator<Element> a=childrena.listIterator(); a.hasNext();)
		{
			Element child = (Element) a.next();
			HttpURLConnection.setFollowRedirects(false);
			URL url;
			try 
			{
				System.out.println("bjkbhjbjk"+"http://"+storeName+".inserts2online.com/"+child.getChildText("detailURL", docNamespace));//("detailURL"));//.getValue());
				System.out.println(child.getTextNormalize());

				url = new URL("http://"+storeName+".inserts2online.com/"+child.getChildText("detailURL",docNamespace));
				
				HttpURLConnection conn = (HttpURLConnection)url.openConnection();
				
			
				conn.setRequestMethod("GET");
				conn.setRequestProperty("Cookie", httpCookie);
				conn.setReadTimeout(5000);
				
				conn.disconnect();
				conn.connect();
				
				System.out.println(conn.getResponseCode());
				if (conn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP || httpCookie.equals("GIVE ME COOKIE"))
				{

					httpCookie = getHTTPCookie(url);
					sendHTTPCookie(httpCookie);
					a.previous();
					System.out.println("PREVIOUS");
					continue;
				}
				
				if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) 
				{		
					System.out.println("");
					//ArrayList<Element> categories = (ArrayList<Element>) child.getChildren("category",docNamespace);
					List<Element> categories = child.getChildren("category",docNamespace);
					
					//BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
					
					Element detailData = getElementWithDetailData(conn, categories);
					
					root.addContent(detailData.setNamespace(outputNS));
					System.out.println(detailData);
						
							
					
				} 
				String outputFile = "." + sep + "detailXml.xml";
				FileOutputStream fos = new FileOutputStream(outputFile);
				xmlOut.setFormat(Format.getPrettyFormat());
				outDoc.setRootElement(root);
				xmlOut.output(outDoc, fos);

				return outputFile;
			
			} catch (MalformedURLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}finally
			{
				
			}
			
			
		}
		
		
		return null;
	}

	private Element getElementWithDetailData(HttpURLConnection conn,
			List<Element> categories) {
		// TODO Auto-generated method stub
		Element returnElement = new Element("Item" );
		//String output="P:\\Parse\\Parse\\Test_Output\\TestOutput.xml";
		Namespace outputNS= Namespace.getNamespace("tns", constants.XML_Namespaces.ItemDetailOutputFormat);
		try
		{
			NodeFilter [] filters ={new HasAttributeFilter("class", VeritasHtmlConstants.itemNameDetailCssClass),
									new HasAttributeFilter("class",VeritasHtmlConstants.itemPriceDetailCssClass),
									new HasAttributeFilter("class",VeritasHtmlConstants.itemDescriptionDetailCssClass),
									new HasAttributeFilter("class",VeritasHtmlConstants.itemPricesGoodDetailCssClass),
									new HasAttributeFilter("class",VeritasHtmlConstants.itemMorePriceDetailCssClass),
									new HasAttributeFilter("alt",VeritasHtmlConstants.largeImageSrcAltText)};
			
			OrFilter orFilter = new OrFilter(filters);
			Parser parse = new Parser(conn);
			
			NodeList list = parse.extractAllNodesThatMatch(orFilter);
			for(int a =0 ; a<list.size(); a++)
			{
				
				TagNode tag = ((TagNode)(list.elementAt(a)));
				
				if(tag.getRawTagName()!=null && tag.getRawTagName().equals("div"))
					returnElement = (processDivTagDetails(tag, returnElement));
				
				else if(tag.getRawTagName()!=null && tag.getRawTagName().equals("img"))
					returnElement.addContent(processImgTagSrc(tag));
				
				
			}
			while(categories.size()>0)
			{
				returnElement.addContent(categories.remove(0).setNamespace(outputNS));
			}
			
		}catch(Exception e){}
		return returnElement;
	}

	private Element processImgTagSrc(TagNode tag) {
		// TODO Auto-generated method stub
		//System.out.println("src: "+tag.getText());
		Element returnElementChild=null;
		
		returnElementChild = new Element("LargeImageSrc");
		Namespace outputNS= Namespace.getNamespace("tns", constants.XML_Namespaces.ItemDetailOutputFormat);
		returnElementChild.setNamespace(outputNS);
		returnElementChild.addContent(tag.getAttribute("src"));
		return returnElementChild;
	}

	private Element processDivTagDetails(TagNode tag, Element returnElement) {
		// TODO Auto-generated method stub
		String className = tag.getAttribute("class");
		Element returnElementChild=null;
		Namespace outputNS= Namespace.getNamespace("tns", constants.XML_Namespaces.ItemDetailOutputFormat);
		
		if(className!=null && className.equals(VeritasHtmlConstants.itemPriceDetailCssClass))
		{
			returnElement.setAttribute("Price", tag.toPlainTextString());
			return returnElement;
		}
		else if(className!=null && className.equals(VeritasHtmlConstants.itemPricesGoodDetailCssClass))
		{
			String [] vaildDates = getVaildDateDetails(tag.toPlainTextString());
			returnElement.setAttribute("ValidStart", vaildDates[0]);
			returnElement.setAttribute("ValidEnd", vaildDates[1]);
			return returnElement;
		}
		else if(className!=null && className.equals(VeritasHtmlConstants.itemNameDetailCssClass))
		{
			returnElement.setAttribute("Name", tag.toPlainTextString());
			return returnElement;
		}
	
		
		else if(className!=null && className.equals(VeritasHtmlConstants.itemDescriptionDetailCssClass))
			returnElementChild = new Element("Description",XML_Namespaces.ItemDetailOutputFormat);
		
		else if(className!=null && className.equals(VeritasHtmlConstants.itemMorePriceDetailCssClass))
			returnElementChild = new Element("SavingsInfo",XML_Namespaces.ItemDetailOutputFormat);
		else
		{
			return null;
			//TODO: Throw a custom exception when unknown class appears
		}
		returnElementChild.setNamespace(outputNS);
		returnElementChild.addContent(tag.toPlainTextString());
		returnElement.addContent(returnElementChild);
		return returnElement;
	}

	private String[] getVaildDateDetails(String text) {
		String [] returnDates = new String [2];
		returnDates[0]="1/1/01";
		returnDates[1]="1/1/01";
		System.out.println("vaild: " +text);
		
		
		Pattern datePattern = Pattern.compile("[1]{0,1}?[0-9]{1}/[1-3]{0,1}?[0-9]{1}/[0-9]{2}");
		Matcher dateMatcher = datePattern.matcher(text);
		SimpleDateFormat parseFormat = new SimpleDateFormat("MM/dd/yy");
		SimpleDateFormat XMLFormat = new SimpleDateFormat("yyyy-MM-dd");
		int a = 0;
		while(dateMatcher.find())
		{
			
			String date = dateMatcher.group();
			
			
			Calendar dateObj = Calendar.getInstance();
			try {
				dateObj.setTime(parseFormat.parse(date));
				returnDates[a] = XMLFormat.format(dateObj.getTime());
				
			} catch (ParseException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			a++;
		}

		return returnDates;
	}

	/*
	 * Precondition:storename, storeId, and adID are set 
	 */
	private void sendHTTPCookie(String httpCookie) throws MalformedURLException {
		// TODO Auto-generated method stub
		
		URL url = new URL("http://" + storeName + ".inserts2online.com/storeReview.jsp?divID=701&drpStoreID="+storeId);//customer_Frame.jsp?divID=701&drpStoreID=00608");
		URL url2 = new URL("http://" + storeName +".inserts2online.com/loadServletXml.jsp?adId="+adId+"&showFlash=false");
		HttpURLConnection conn;
	
		try {
			conn = (HttpURLConnection)url.openConnection();
			conn.setRequestMethod("GET");
			conn.setRequestProperty("Cookie", httpCookie);
			//HttpURLConnection.setFollowRedirects(true);
			conn.disconnect();
			conn.connect();
			conn.disconnect();
			//HttpURLConnection.setFollowRedirects(false);
			System.out.println(conn.getResponseCode());
			
			conn = (HttpURLConnection)url2.openConnection();
			conn.setRequestMethod("GET");
			conn.setRequestProperty("Cookie", httpCookie);
			System.out.println("2: "+conn.getResponseCode());
			
		//System.exit(0);
		}
		catch(Exception e)
		{}
		
	}

	private String getHTTPCookie(URL url){
		// TODO Auto-generated method stub
		do{
		String headerName = "";
		String httpCookie = "";
		HttpURLConnection conn;
		
		
		try {
			conn = (HttpURLConnection)url.openConnection();
			conn.setRequestMethod("GET");
			conn.setRequestProperty("Accept", "*/*");
			conn.setRequestProperty("Accept-Language","en-us;");
			conn.setRequestProperty("UA-CPU","x86");
			conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
			conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; GTB5; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 1.1.4322; .NET CLR 3.5.30729; .NET CLR 3.0.30618)");
			conn.setRequestProperty("Connection",  "Keep-Alive");
			conn.connect();
			
		
			if(conn.getResponseCode() == HttpURLConnection.HTTP_OK || 
					conn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP)
			{
				for (int i=1; (headerName = conn.getHeaderFieldKey(i))!=null; i++) 
				{
				 	//System.out.println(headerName + "||");
					if (headerName.equals("Set-Cookie")) 
					{                  
						httpCookie += conn.getHeaderField(i)+";";   
					}            
				}
				httpCookie = Pattern.compile("Path.*?;",Pattern.CASE_INSENSITIVE).matcher(httpCookie).replaceAll("");

				return httpCookie;
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
/*		String cookie = ""; 
		URL url;
		try {
			url = new URL("http://"+storeName+".inserts2online.com/");
		
		HttpURLConnection conn;
		
		conn = (HttpURLConnection)url.openConnection();
		
		
		conn.setRequestMethod("GET");
		conn.connect();
		if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
			
			cookie = conn.getHeaderField("Set-Cookie");
			String headerName=null;
			for (int i=1; (headerName = conn.getHeaderFieldKey(i))!=null; i++) 
			{
			 	System.out.println(headerName + "||");
				if (headerName.equals("Set-Cookie")) 
				{                  
					cookie += conn.getHeaderField(i);   
				}            
			}
			System.out.println(cookie);
			BufferedReader in = new BufferedReader	(new InputStreamReader(conn.getInputStream()));
			String line = null;
			while((line = in.readLine())!=null)
			{	
				if(line.startsWith("Set-Cookie"))
					cookie+= line.substring(cookie.indexOf(':')+1,cookie.indexOf(';')+1);
			}
		}
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.out.println("vertisParse.getHTTPCookie");
		} catch (ProtocolException e){
			e.printStackTrace();
			System.out.println("vertisParse.getHTTPCookie");
		}catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.out.println("vertisParse.getHTTPCookie");
		} */
		}while(true);
	}

}
